#!/bin/env python
# Copyright (c) Lawrence Livermore National Security, LLC and other VisIt
# Project developers.  See the top-level LICENSE file for dates and other
# details.  No copyright assignment is required to contribute to VisIt.
"""
VisIt's test suite.
"""
# ----------------------------------------------------------------------------
#  Modifications:
#    Cyrus Harrison, Wed Sep  7 11:34:36 PDT 2022
#    Refactored to module.
#
# ----------------------------------------------------------------------------

import sys
import os
import math
import glob
import re
import shutil
import subprocess
import multiprocessing
import time
import math
import datetime
import pickle
import json
import traceback


from os.path import join as pjoin
from optparse import OptionParser

from .visit_test_common import *
from .visit_test_reports import *
from .visit_test_ctest import *


def known_mode_keys():
    return ['serial','parallel','scalable','dlb','pdb','hdf5','icet']

# ----------------------------------------------------------------------------
#  Method: visit_root
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
# ----------------------------------------------------------------------------
def visit_root():
    return abs_path(test_path(),"../..")


# ----------------------------------------------------------------------------
#  Method: test_path
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
# ----------------------------------------------------------------------------
def test_path():
    return os.path.abspath(os.path.split(__file__)[0])


# ----------------------------------------------------------------------------
#  Method: check_skip
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
#
# Modifications:
#  Cyrus Harrison, Tue May 28 12:05:45 PDT 2013
#  Add support for platform based skips.
#
#  Kathleen Biagas, Thu Nov  8 10:30:36 PST 2018
#  Assume entire category is skipped it 'file' key is missing.
#
# ----------------------------------------------------------------------------
def check_skip(skip_list,test_modes,test_cat,test_file):
    if skip_list is None:
        return False
    # look for modes that match
    for v in skip_list['skip_list']:
        if v['mode'] == test_modes:
            for test in v['tests']:
                # check for platform restrictions
                if "platform" in test:
                    tplat = test["platform"].lower()
                    splat = sys.platform.lower()
                    # win,linux,osx
                    # ignore this entry if we are on the wrong platform
                    # else, use std logic
                    if not splat.startswith(tplat):
                        continue
                if test['category'] == test_cat:
                    if "file" in test:
                        if test['file'] == test_file:
                            if "cases" not in test:
                            # skip the entire file if
                            # there are no specific cases
                                return True
                    else:
                        # skip the entire category
                        return True
    return False
# ----------------------------------------------------------------------------
#  Method: parse_test_specific_vargs
#
#  Programmer: Cyrus Harrison
#  Date:       Tues April 29, 2014
# ----------------------------------------------------------------------------
def parse_test_specific_vargs(test_file):
    lines = [l.strip() for l in open(test_file).readlines()]
    # check for pattern "# ... VARGS: {visit command line args}
    vargs = [l[1:] for l in lines if len(l) > 0 and l[0] == "#" and l.count("VARGS:")  == 1]
    vargs = " ".join(vargs)
    vargs = vargs.replace("VARGS:","")
    return vargs

# ----------------------------------------------------------------------------
#  Method: parse_test_specific_limit
#
#  Programmer: Kathleen Biagas
#  Date:       Thu Nov 8, 2018
# ----------------------------------------------------------------------------
def parse_test_specific_limit(test_file):
    lines = [l.strip() for l in open(test_file).readlines()]
    # check for pattern "# ... LIMIT: {limit}
    limline = [l[1:] for l in lines if len(l) > 0 and l[0] == "#" and l.count("LIMIT:")  == 1]
    if len(limline) > 0:
        limline = " ".join(limline)
        limline = limline.replace("LIMIT:","")
        return int(limline)
    return -1

# ----------------------------------------------------------------------------
#  Method: launch_visit_test
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
#
#  Modifications:
#    Brad Whitlock, Thu Nov  7 14:00:28 PST 2013
#    Pass width and height to test.
#
#    Kathleen Biagas, Thu Feb  6 14:08:00 PST 2014
#    Pass 'ctest' to test.
#
#    Kathleen Biagas, Fri May 16 15:23:13 PDT 2014
#    Set up sim_dir based off executable location.
#
#    Burlen Loring, Mon May 26 15:36:26 PDT 2014
#    Addedd command line option to use threshold based image diff
#
#    Eric Brugger, Fri Aug 15 10:04:27 PDT 2014
#    I added the ability to specify the parallel launch method.
#
#    Kathleen Biagas, Thu Sep 4 16:45:39 MST 2014
#    Use exe path only for sim_dir on Windows.
#
#    Burlen Loring, Wed Oct 21 15:44:24 PDT 2015
#    Added an option (--display-failed) to display current,
#    baseline, and diff in a popup window as the test runs.
#
#    Kathleen Biagas, Thu Nov  8 10:31:51 PST 2018
#    Added src_dir and cmake_cmd, for plugin-vs-install tests.
#    If test has specified its own 'LIMIT' use it for the kill limit.
#
#    Kathleen Biagas, Thu Mar 4, 2021
#    Utilize cleanup_delay when cleaning up run_dir when test finishes.
#    Fixes issue of run_dir not being completely removed when run on Windows.
#
# ----------------------------------------------------------------------------
def launch_visit_test(args):
    """
    Runs a single VisIt test.
    """
    idx  =  args[0]
    test =  args[1]
    opts =  args[2]
    top_dir     = visit_root()
    test_script = abs_path(test_path(),"visit_test_main.py")
    test_dir, test_file = os.path.split(test)
    test_cat  = os.path.split(test_dir)[1]
    test_base = os.path.splitext(test_file)[0]
    rcmd  =  opts["executable"] + " "
    rcmd +=  opts["vargs"] + " "
    # check for vargs embedded in the test file header
    rcmd +=  parse_test_specific_vargs(test) + " "
    rcmd +=  "-exec-timeout %d -idle-timeout %d " % (opts["limit"],opts["limit"])
    rcmd +=  "-numrestarts 0 "
    if not opts["interactive"]:
        rcmd +=  "-nowin "
    if not opts["use_pil"]:
        rcmd +=  "-noPIL "
    if not opts["no_timings"]:
        rcmd +=  "-timing"
    rcmd += " -cli "
    cfile = pjoin(test_dir,test_base + ".config")
    if os.path.isfile(cfile):
        rcmd += "-config " + cfile + " "
    else:
        rcmd += "-noconfig "
    rcmd += "-geometry %dx%d+32+32 " % (opts["width"],opts["height"])
    rcmd += " -s %s " % os.path.abspath(test_script)
    modes_list = opts["modes"].split(",")
    if "dlb" in modes_list:
        rcmd += " -allowdynamic "
    if "icet" in modes_list:
        rcmd += " -icet "
    modes = ""
    if opts["verbose"]:
        rcmd += " -verbose "
    fuzzy = opts["fuzzy"]
    if "serial" in modes_list:
        modes = "serial"
    else:
        if "scalable" in modes_list:
            modes = "scalable"
            if not opts["no_fuzzy"]:
                fuzzy = True
        if "parallel" in modes_list:
            if len(modes) > 0:
                modes +=","
            modes +="parallel"
        if "icet" in modes_list:
            if len(modes) > 0:
                modes +=","
            modes +="icet"
    if "pdb" in modes_list:
        if modes == "":
            modes = "pdb"
        else:
            modes += ",pdb"
    run_dir = pjoin(opts["result_dir"],"_run","_%s_%s" % (test_cat, test_base))
    # set opts vars
    tparams = {}
    tparams["script"]         = test
    tparams["category"]       = test_cat
    tparams["name"]           = test_base
    tparams["file"]           = test_file
    tparams["modes"]          = modes
    tparams["run_dir"]        = run_dir
    tparams["result_dir"]     = opts["result_dir"]
    tparams["fuzzy_match"]    = fuzzy
    tparams["skip_file"]      = None
    tparams["interactive"]    = opts["interactive"]
    tparams["use_pil"]        = opts["use_pil"]
    tparams["threshold_diff"] = opts["threshold_diff"]
    tparams["threshold_error"]= opts["threshold_error"]
    tparams["pixdiff"]        = opts["pixdiff"]
    tparams["avgdiff"]        = opts["avgdiff"]
    tparams["numdiff"]        = opts["numdiff"]
    tparams["top_dir"]        = top_dir
    tparams["data_dir"]       = opts["data_dir"]
    tparams["src_dir"]        = opts["src_dir"]
    tparams["data_host"]      = opts["data_host"]
    tparams["baseline_dir"]   = opts["baseline_dir"]
    tparams["tests_dir"]      = opts["tests_dir"]
    tparams["visit_bin"]      = opts["executable"]
    tparams["width"]          = opts["width"]
    tparams["height"]         = opts["height"]
    tparams["ctest"]          = opts["ctest"]
    tparams["display_failed"] = opts["display_failed"]
    tparams["parallel_launch"]= opts["parallel_launch"]
    tparams["host_profile_dir"]   = opts["host_profile_dir"]
    tparams["sessionfiles"]   = opts["sessionfiles"]
    tparams["cmake_cmd"]      = opts["cmake_cmd"]
    tparams["clargs"]         = json.dumps(sys.argv)

    exe_dir, exe_file = os.path.split(tparams["visit_bin"])
    if sys.platform.startswith("win"):
        tparams["sim_dir"] = os.path.abspath(exe_dir)
    else:
        tparams["sim_dir"] = os.path.abspath(os.path.join(exe_dir, ".."))

    if not opts["no_skip"]:
        tparams["skip_file"]  = opts["skip_file"]
    skip  =  check_skip(opts["skip_list"],modes,test_cat,test_file)
    if skip:
        Log("[Skipping: %s/%s (found in skip list)]" % (test_cat,test_file))
        result = TestScriptResult(idx,
                                  test_cat,
                                  test_base,
                                  test_file,
                                  116, # full skip return code
                                  0,
                                  0)
    else:
        Log("[Launching: %s/%s]" % (test_cat,test_file))
        # run the test in a unique sub dir
        if os.path.isdir(run_dir):
            try:
                shutil.rmtree(run_dir)
            except OSError as e:
                Log("<Error Cleaning up Script Run Directory before launch> %s" % run_dir)
        os.mkdir(run_dir)
        pfile = open(pjoin(run_dir,"params.json"),"w")
        pfile.write("%s" % json_dumps(tparams))
        pfile.close()
        rcmd += " --params=%s" % os.path.abspath(pjoin(run_dir,"params.json"))
        # get start timestamp
        stime = time.time()
        # change to working dir to our run dir
        curr_dir = os.getcwd()
        os.chdir(run_dir)
        rcode = 0
        test_specific_limit = parse_test_specific_limit(test)
        if test_specific_limit != -1:
            use_limit = test_specific_limit
        else:
            use_limit = opts["limit"]
        sexe_res = sexe(rcmd,
                        suppress_output=(not (opts["verbose"] or opts["less_verbose"])),
                        echo=opts["verbose"],
                        timeout=use_limit * 1.1) # proc kill switch at 110% of the selected timeout
        json_res_file = pjoin(opts["result_dir"],"json","%s_%s.json" %(test_cat,test_base))
        if os.path.isfile(json_res_file):
            results = json_load(json_res_file)
            if "result_code" in results:
                rcode = results["result_code"]
            # os.mkdir(run_dir)
        if sexe_res["killed"]:
            Log("<Limit killed> %s" % os.path.abspath(test))
            # wait for process chain to die
            time.sleep(1)
        # get end timestamp
        etime = time.time()
        dtime = math.ceil(etime - stime)
        # wait for any logs to flush
        time.sleep(1)
        # change back to prev working dir
        os.chdir(curr_dir)
        # check for core files
        ncores_files = process_cores(run_dir,opts["result_dir"],test_cat,test_base)
        # move logs and timings to the html output dir
        process_runtime_logs(run_dir,opts["result_dir"],test_cat,test_base)
        # parse any output files
        result = TestScriptResult(idx,
                                  test_cat,
                                  test_base,
                                  test_file,
                                  rcode,
                                  ncores_files,
                                  dtime)
        if opts["cleanup"]:
            time.sleep(opts["cleanup_delay"])
            try:
                shutil.rmtree(run_dir)
            except OSError as e:
                Log("<Error Removing Script Run Directory> %s" % run_dir)
    log_test_result(opts["result_dir"],result)
    return result


# ----------------------------------------------------------------------------
#  Method: process_cores
#
#  Programmer: Cyrus Harrison
#  Date:       Thu Aug  9 2012
# ----------------------------------------------------------------------------
def process_cores(run_dir,res_dir,test_cat,test_name):
    MAX_CORE_SIZE = 262144000 # 250 mbytes
    core_files    = glob.glob(pjoin(run_dir,"*core*"))
    num_cores     = len(core_files)
    if num_cores > 0:
        cores_base_dir = pjoin(res_dir,"cores")
        cores_cat_dir  = pjoin(cores_base_dir,test_cat)
        cores_test_dir = pjoin(cores_cat_dir,test_name)
        # make sure the log dirs exists
        for dname in [cores_base_dir, cores_cat_dir, cores_test_dir]:
            if not os.path.isdir(dname):
                os.mkdir(dname)
        # move any cores into this dir
        for cfile in core_files:
            # make sure we don't copy huge core files.
            csize = os.path.getsize(cfile)
            if csize < MAX_CORE_SIZE:
                shutil.copy2(cfile, cores_test_dir)
    return num_cores


# ----------------------------------------------------------------------------
#  Method: process_runtime_logs
#
#  Programmer: Cyrus Harrison
#  Date:       Thu Aug  9 2012
# ----------------------------------------------------------------------------
def process_runtime_logs(run_dir,res_dir,test_cat,test_name):
    rlog_files = glob.glob(pjoin(run_dir,"*vlog"))
    rlog_files.extend(glob.glob(pjoin(run_dir,"*.timings")))
    if len(rlog_files) > 0:
        logs_base_dir = pjoin(res_dir,"logs")
        logs_cat_dir  = pjoin(logs_base_dir,test_cat)
        logs_test_dir = pjoin(logs_cat_dir ,test_name)
        # make sure the log dirs exists
        for dname in [logs_base_dir, logs_cat_dir, logs_test_dir]:
            if not os.path.isdir(dname):
                os.mkdir(dname)
        for rlog in rlog_files:
            shutil.copy2(rlog, logs_test_dir)


# ----------------------------------------------------------------------------
#  Method: log_test_result
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
# ----------------------------------------------------------------------------
def log_test_result(result_dir,result):
    Log(result.message())
    HTMLIndex(result_dir).add_result(result)
    JSONIndex(pjoin(result_dir,"results.json")).add_result(result.to_dict())


# ----------------------------------------------------------------------------
#  Method: default_suite_options
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 8 2013
#
#  Modifications:
#    Eric Brugger, Fri Aug 15 10:04:27 PDT 2014
#    I added the ability to specify the parallel launch method.
#
#    Brad Whitlock, Mon Dec 15 15:42:32 PST 2014
#    Added --data-host, --host-profile-dir.
#
#    Burlen Loring, Wed Oct 21 15:44:24 PDT 2015
#    Added an option (--display-failed) to display current,
#    baseline, and diff in a popup window as the test runs.
#
#    Mark C. Miller, Tue Sep  6 18:54:31 PDT 2016
#    Added sessionfiles option to rigoursly test session files
#
#    Kathleen Biagas, Thu Nov  8 10:33:45 PST 2018
#    Added src_dir and cmake_cmd.
#
#    Eric Brugger, Wed Dec  5 13:05:18 PST 2018
#    Changed the definition of tests_dir_def to the new location of the
#    test directory.
#
#    Kathleen Biagas, Thu Dec 13 10:51:54 PST 2018
#    Changed definition of data_dir to new location.  Assumes in-src build,
#    which is used for nightly regression tests.
#
#    Cyrus Harrison, Mon Jan 29 16:12:16 PST 2024
#    Change test limit to 800 (an experiment for nightly regression suite )
#
# ----------------------------------------------------------------------------
def default_suite_options():
    data_dir_def    = abs_path(visit_root(),"src","testdata")
    base_dir_def    = abs_path(visit_root(),"test","baseline")
    tests_dir_def   = abs_path(visit_root(),"src","test","tests")
    visit_exe_def   = abs_path(visit_root(),"src","bin","visit")
    src_dir_def     = abs_path(visit_root(),"src")
    skip_def        = pjoin(test_path(),"skip.json")
    # Set nprocs_def to 1, since multi-proc test mode seems to result in
    # crossed streams. In the past we have used: multiprocessing.cpu_count()
    nprocs_def      = 1
    opts_full_defs = {
                      "use_pil":True,
                      "threshold_diff":False,
                      "threshold_error":{},
                      "src_dir":      src_dir_def,
                      "data_dir":     data_dir_def,
                      "baseline_dir": base_dir_def,
                      "tests_dir":    tests_dir_def,
                      "result_dir":   test_path(),
                      "post":False,
                      "verbose":False,
                      "less_verbose":False,
                      "quiet":False,
                      "width": 300,
                      "height":300,
                      "modes":"serial",
                      "classes":"nightly",
                      "limit":800,
                      "skip_file":skip_def,
                      "no_skip":False,
                      "check_data":True,
                      "fuzzy":False,
                      "no_fuzzy":False,
                      "cleanup":True,
                      "cleanup_delay":10,
                      "executable":visit_exe_def,
                      "data_host":"localhost",
                      "interactive":False,
                      "pixdiff":0.0,
                      "avgdiff":0.0,
                      "numdiff":0.0,
                      "vargs": "",
                      "host_profile_dir": "",
                      "retry":False,
                      "index":None,
                      "timeout":9000,
                      "nprocs":nprocs_def,
                      "ctest":False,
                      "display_failed":False,
                      "parallel_launch":"mpirun",
                      "sessionfiles":False,
                      "cmake_cmd":"cmake",
                      "no_timings":False,
                      "rsync_post":None}
    return opts_full_defs

def finalize_options(opts):
    opts["executable"]   = abs_path(opts["executable"])
    opts["result_dir"]   = abs_path(opts["result_dir"])
    opts["src_dir"]      = abs_path(opts["src_dir"])
    opts["data_dir"]     = abs_path(opts["data_dir"])
    opts["tests_dir"]    = abs_path(opts["tests_dir"])
    opts["baseline_dir"] = abs_path(opts["baseline_dir"])
    if isinstance(opts["classes"],str):
        opts["classes"]  = opts["classes"].split(",")
    opts["skip_list"]    = None

    if not opts["skip_file"] is None:
        if not os.path.isfile(opts["skip_file"]):
            opts["skip_file"] = abs_path(opts["tests_dir"], "../", "skip.json")
        if os.path.isfile(opts["skip_file"]):
            try:
                if opts["no_skip"] == False:
                    opts["skip_list"] = json_load(opts["skip_file"])
            except:
                opts["skip_list"] = None


# ----------------------------------------------------------------------------
#  Method: parse_args
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
#
#  Modifications:
#    Eric Brugger, Fri Aug 15 10:04:27 PDT 2014
#    I added the ability to specify the parallel launch method.
#
#    Kathleen Biagas, Wed Nov 5 14:32:21 PST 2014
#    On windows, glob any '*.py' tests names.
#
#    Matthew Wheeler, Mon Dec 15 12:56:00 GMT 2014
#    Changed pixdiff % to be a float rather than an int
#
#    Burlen Loring, Wed Oct 21 15:44:24 PDT 2015
#    Added an option (--display-failed) to display current,
#    baseline, and diff in a popup window as the test runs.
#
#    Mark C. Miller, Tue Sep  6 18:54:31 PDT 2016
#    Added sessionfiles option to rigoursly test session files
#
#    Kathleen Biagas, Thu Nov  8 10:34:27 PST 2018
#    Added '--src' for specifying src_dir, and --cmake for specifying
#    cmake_cmd, used for plugin-vs-install tests.
#
#    Kathleen Biagas, Wed Dec 18 17:22:59 MST 2019
#    For windows, move the glob of '*.py' tests name to after full-path
#    expansion in main.
#
#    Mark C. Miller, Thu Nov 14 18:54:01 PST 2024
#    Improve usage info for --fuzzy, --pixdiff and --avgdiff.
# ----------------------------------------------------------------------------
def parse_args():
    """
    Parses arguments to runtest.
    """
    parser = OptionParser()
    defs = default_suite_options()
    parser.add_option("-r",
                      "--run-only",
                      dest="use_pil",
                      default=True,
                      action="store_false",
                      help="no image differencing (no PIL)")
    parser.add_option( "--src",
                      dest="src_dir",
                      default=defs["src_dir"],
                      help="path to src directory [default=%s]" % defs["src_dir"])
    parser.add_option("-d",
                      "--data-dir",
                      dest="data_dir",
                      default=defs["data_dir"],
                      help="path to data directory [default=%s]" % defs["data_dir"])
    parser.add_option("-b",
                      "--baseline-dir",
                      dest="baseline_dir",
                      default=defs["baseline_dir"],
                      help="path to baseline directory [default=%s]" % defs["baseline_dir"])
    parser.add_option("-t",
                      "--tests-dir",
                      dest="tests_dir",
                      default=defs["tests_dir"],
                      help="path to tests directory [default=%s]" % defs["tests_dir"])
    parser.add_option("-o",
                      "--output-dir",
                      dest="result_dir",
                      default=defs["result_dir"],
                      help="path to output directory [default=%s]" % defs["result_dir"])
    parser.add_option("-p",
                      "--post",
                      default=defs["post"],
                      action="store_true",
                      help="post results to web")
    parser.add_option("-q",
                      "--quiet",
                      default=defs["quiet"],
                      action="store_true",
                      help="suppress all text output")
    parser.add_option("-v",
                      "--verbose",
                      default=defs["verbose"],
                      action="store_true",
                      help="extra test output")
    parser.add_option("--lessverbose",
                      default=defs["less_verbose"],
                      dest="less_verbose",
                      action="store_true",
                      help="extra test output without progress messages")
    parser.add_option("--width",
                      type="int",
                      default=defs["width"],
                      help="set image width")
    parser.add_option("--height",
                      type="int",
                      default=defs["height"],
                      help="set image height")
    parser.add_option("-m",
                      "--modes",
                      default=defs["modes"],
                      help="specify mode keys used to run tests "
                           "choose from %s and comma-separated combinations such as "
                           "scalable,parallel. Any unrecognized mode keys are passed "
                           "through uninterpreted. [default serial]"%known_mode_keys())
    parser.add_option("-c",
                      "--classes",
                      default=defs["classes"],
                      help="list of classes [choose from 'nightly']")
    parser.add_option("-l",
                      "--limit",
                      type="int",
                      default=defs["limit"],
                      help="set maximum elapsed run-time for each test")
    parser.add_option("-s",
                      "--skiplist",
                      dest="skip_file",
                      default=defs["skip_file"],
                      help="specify a skip list file [default=%s]" % defs["skip_file"])
    parser.add_option("--no-skiplist",
                      dest="no_skip",
                      default=defs["no_skip"],
                      action = "store_true",
                      help="Do not use a skip list file")
    parser.add_option("--no-data-check",
                      dest="check_data",
                      default=defs["check_data"],
                      action = "store_false",
                      help="Skip build sanity check on input data files")
    parser.add_option("--fuzzy",
                      dest="fuzzy",
                      default=defs["fuzzy"],
                      action = "store_true",
                      help="Shorthand to set minimum fuzzy thresholds to "
                           "--pixdiff 2 --avgdiff 1 [default=False].")
    parser.add_option("--no-fuzzy",
                      dest="no_fuzzy",
                      default=defs["no_fuzzy"],
                      action = "store_true",
                      help="Do not use fuzzy image matching for scalable mode"),
    parser.add_option("--no-cleanup",
                      dest="cleanup",
                      default=defs["cleanup"],
                      action = "store_false",
                      help="Do not remove the _run directory"),
    parser.add_option("--cleanup-delay",
                      dest="cleanup_delay",
                      default=defs["cleanup_delay"],
                      type="int",
                      help="# of seconds to wait for processes to finish "
                           "before cleaning up. [default=%d]" % defs["cleanup_delay"]),
    parser.add_option("-e",
                      "--executable",
                      default=defs["executable"],
                      help="specify executable version of visit to run. "
                           "For example, use \"-e '/usr/gapps/visit/bin/visit -v "
                           " [default = %s]" % defs["executable"])
    parser.add_option("--data-host",
                      dest="data_host",
                      default=defs["data_host"],
                      help="Specify remote host to use for data and compute engine. "
                           "For example, use \"--data-host vulcan.llnl.gov "
                           " [default = %s]" % defs["data_host"])
    parser.add_option("-i",
                      "--interactive",
                      action = "store_true",
                      default=defs["interactive"],
                      help="don't redirect test .py file into visit."
                           "Just bring up CLI and let user Source() "
                           "the .py file explicitly.")
    parser.add_option("--pixdiff",
                      type="float",
                      default=defs["pixdiff"],
                      help="Allowed % of pixels different [default = 0.0%]. "
                           "'--pixdiff 6.3', for example, means 6.3% of pixels can differ. "
                           "When both pixdiff and avgdiff are specified, avgdiff is considered "
                           "only if pixdiff has not been exceeded.")
    parser.add_option("--avgdiff",
                      type="float",
                      default=defs["avgdiff"],
                      help="Allowed mean grayscale diff [default = 0]. "
                           "When both pixdiff and avgdiff are specified, avgdiff is considered "
                           "only if pixdiff has not been exceeded.")
    parser.add_option("--numdiff",
                      type="float",
                      default=defs["numdiff"],
                      help="Allowed relative numerical difference in text "
                           "results [default = 0.0]")
    parser.add_option("--vargs",
                      default=defs["vargs"],
                      help="arguments to pass directly to VisIt "
                           "(surround them \" or ')")
    parser.add_option("--host-profile-dir",
                      default=defs["host_profile_dir"],
                      help="Specify a directory from which host profiles will be read.")
    parser.add_option("--retry",
                      default=defs["retry"],
                      action="store_true",
                      help="retry failed test cases from last index file")
    parser.add_option("--index",
                      dest="index",
                      default=defs["index"],
                      help="load test cases from a specific index file")
    parser.add_option("--timeout",
                      type="int",
                      default=defs["timeout"], # total timeout of two hours
                      help="total test suite timeout in seconds [default = 9000]")
    parser.add_option("-n","--num-processes",
                      dest="nprocs",
                      type=int,
                      default=defs["nprocs"],
                      help="number of tests to launch simultaneously [default =%d]" % defs["nprocs"])
    parser.add_option("--ctest",
                      default=defs["ctest"],
                      action="store_true",
                      help="generate ctest compatible output")
    parser.add_option("--threshold-diff",
                      default=defs["threshold_diff"],
                      dest="threshold_diff",
                      action="store_true",
                      help="use threshold based image diff")
    parser.add_option("--threshold-error",
                      default=defs["threshold_error"],
                      dest="threshold_error",
                      type=str,
                      action='callback',
                      callback=ParseThresholdOverride,
                      help="Per case overide of the max allowable error for threshold based image diff")
    parser.add_option("--display-failed",
                      default=defs["display_failed"],
                      action="store_true",
                      help="use image magick to display the current baseline and diff for each failure")
    parser.add_option("--parallel-launch",
                      default=defs["parallel_launch"],
                      help="specify the parallel launch method. "
                           "Options are mpirun and srun, or a space-separated "
                           "list of parallel launch options.")
    parser.add_option("--sessionfiles",
                      dest="sessionfiles",
                      action = "store_true",
                      default=defs["sessionfiles"],
                      help="Rigoursly test session files by saving a session, "
                           "validating the XML and then perturbing and "
                           "restoring the session prior to each image save.")
    parser.add_option("--no-timings",
                      dest="no_timings",
                      default=defs["no_timings"],
                      action = "store_true",
                      help="Do not generate timing files.")
    parser.add_option("--rsync-post",
                      default=defs["rsync_post"],
                      help="Post results via rsync")
    parser.add_option( "--cmake",
                      dest="cmake_cmd",
                      default=defs["cmake_cmd"],
                      help="path to cmake executable [default=%s]" % defs["cmake_cmd"])

    # parse args
    opts, tests = parser.parse_args()
    # note: we want a dict b/c the values could be passed without using optparse
    opts = vars(opts)
    return opts, tests

# ----------------------------------------------------------------------------
#  Method: ParseThresholdOverride
#
#  Programmer: Burlen Loring
#  Date:       Tue May 27 11:11:22 PDT 2014
# ----------------------------------------------------------------------------
def ParseThresholdOverride(option, opt, value, parser):
    """
    Convert threshold overrides encodeed in a string like
    --threshold-error=a:b,c:d to a dictionary. Parse
    errors are intentionally fatal.
    """
    d={}
    for pair in value.split(','):
        k,v = pair.split(':')
        d[k] = float(v)
    setattr(parser.values, option.dest, d)

# ----------------------------------------------------------------------------
#  Method: find_test_cases
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
# ----------------------------------------------------------------------------
def find_test_cases(tests_dir,test_classes):
    """
    Finds test suite scripts.
    """
    Log("[Finding test scripts]")
    if len(test_classes)> 0:
        Log("[Test Classes: %s]" % str(test_classes))
    res    = []
    tfiles = glob.glob(pjoin(tests_dir,"*","*.py"))
    tfiles = [ os.path.abspath(tf) for tf in tfiles]
    if len(test_classes) == 0 or test_classes == "ANY":
        res = tfiles
    else:
        for tf in tfiles:
            test_lines = open(tf).readlines()
            match = False
            for l in test_lines:
                if l.count("#  CLASSES:"):
                    for c in test_classes:
                        if l.count(c):
                            match = True
            if match:
                res.append(tf)
            else:
                Log("Excluding: %s" %tf)
    return res

# ----------------------------------------------------------------------------
#  Method: load_test_cases_from_index
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
# ----------------------------------------------------------------------------
def load_test_cases_from_index(tests_dir,result_idx,only_failures=False):
    """
    Finds test suite scripts.
    """
    Log("[Loading test cases from index file = %s)]" % result_idx)
    res = []
    tests = JSONIndex.load_results(result_idx)
    for t in tests:
        tsr = TestSuiteResult(**t)
        if only_failures and tsr.error():
            res.append(pjoin(tests_dir,tsr.category,tsr.file))
        else:
            res.append(pjoin(tests_dir,tsr.category,tsr.file))
    return res

# ----------------------------------------------------------------------------
#  Method: prepare_result_dirs
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
# ----------------------------------------------------------------------------
def prepare_result_dirs(res_dir=None):
    """
    Creates output directories for test suite results
    """
    if res_dir is None:
        res_dir = test_path()
    else:
        if not os.path.isdir(res_dir):
            os.mkdir(res_dir)
    run_dir = pjoin(res_dir,"_run")
    if not os.path.isdir(run_dir):
        os.mkdir(run_dir)
    for d in ["html","json","current","diff"]:
        full_dir = pjoin(res_dir,d)
        if os.path.isdir(full_dir):
            shutil.rmtree(full_dir)
        os.mkdir(full_dir)

# ----------------------------------------------------------------------------
#  Method: prepare_data_dir
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
#
#  Modifications:
#    Kathleen Biagas, Mon Jan 27 13:02:45 MST 2014
#    Change 'make test' to 'make testdata'.
# ----------------------------------------------------------------------------
def prepare_data_dir(data_dir):
    """
    Simple check of test data.
    """
    dfile = pjoin(data_dir,"silo_hdf5_test_data","globe.silo")
    Log("[Checking test data]")
    if not os.path.isfile(dfile):
        Log("[Rebuilding test data based "
            "on the fact that '%s' doesn't exist]" % dfile)
        if sys.platform.startswith("win"):
            # TODO: What should we do on windows if the data doesn't exist?
            pass
        else:
            cwd = os.getcwd()
            os.chdir(data_dir)
            cmd = "make testdata"
            sexe(cmd)
            os.chdir(cwd)
    else:
        Log("[Test data file '%s' exists.]" % dfile)

# ----------------------------------------------------------------------------
#  Method: cleanup
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
# ----------------------------------------------------------------------------
def cleanup(res_dir,delay):
    Log("[Cleanup: Waiting for delayed writes]")
    time.sleep(delay)
    Log("[Cleanup: Removing _run directory]")
    run_dir = pjoin(res_dir,"_run")
    try:
        shutil.rmtree(run_dir)
    except OSError as e:
        Log("<Error Removing _run Directory> %s" % run_dir)

# ----------------------------------------------------------------------------
#  Method: launch_tests
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
# ----------------------------------------------------------------------------
def launch_tests(opts,tests):
    error = False
    results = []
    ntests = len(tests)
    if ntests == 1:
        Log("[Running %d test case]" % ntests)
    else:
        Log("[Running %d test cases]" % ntests)
    if opts["no_skip"] == False and os.path.isfile(opts["skip_file"]):
        Log("[Using skip list file: '%s']" % opts["skip_file"])
    test_args = [(idx,tests[idx],opts) for idx in range(ntests)]
    # save the input list
    test_list = json_dumps([(idx,tests[idx]) for idx in range(ntests)])
    open(pjoin(opts["result_dir"],"tests.json"),"w").write(test_list)
    if len(test_args) < opts["nprocs"]:
        opts["nprocs"] = len(test_args)
    if opts["nprocs"] < 2:
        Log("[Using %s test process]" % opts["nprocs"])
        for args in test_args:
            results.append(launch_visit_test(args))
    else:
        Log("[Using %s test processes]" % opts["nprocs"])
        pool = multiprocessing.Pool(processes=opts["nprocs"])
        p = pool.map_async(launch_visit_test,test_args)
        try:
            #results = p.get(0xFFFF) # used to make sure ctl-c doesn't hang
            results = p.get(opts["timeout"]) # used to make sure ctl-c doesn't hang
        except KeyboardInterrupt:
            Log("<Test Suite Run aborted by user keyboard interrupt.>")
            pool.terminate()
            error = True
        except Exception as e:
            traceback.print_exc()
            Log("<Unknown Exception>:%s" % str(e))
            pool.terminate()
            error = True
    if len(results) != len(test_list):
        for t in test_args:
            if not t[0] in [ r.index for r in results]:
                test_dir, test_file = os.path.split(t[1])
                test_cat  = os.path.split(test_dir)[1]
                test_base = os.path.splitext(test_file)[0]
                Log("<Missing result from test %s/%s, assuming Killed>" % (test_cat,test_base))
                res  = TestScriptResult(t[0],
                                        test_cat,
                                        test_base,
                                        test_file,
                                        115, # killed return code
                                        0,
                                        0)
                results.append(res)
    error = error or True in [r.error() for r in results]
    return error, results

# ----------------------------------------------------------------------------
#  Method: rsync_post
#
#  Programmer: Cyrus Harrison
#  Date:       ue Nov 25 11:23:12 PST 2014
#
#  Modifications:
# ----------------------------------------------------------------------------
def rsync_post(src_dir,rsync_dest):
    rsync_dest = pjoin(rsync_dest,timestamp(sep="_"))
    rsync_cmd =  "rsync -vur %s/ %s" % (src_dir,rsync_dest)
    Log("[rsyncing results to %s]" % rsync_dest)
    exe_res = sexe(rsync_cmd,
                   suppress_output=(not (opts["verbose"] or opts["less_verbose"])),
                    echo=opts["verbose"])


# ----------------------------------------------------------------------------
#  Method: resolve_test_paths
#  Purpose: Help resolve tests paths
#
#  Programmer: Cyrus Harrison + Kathleen Biagas
#  Date:       Fri Apr 10 09:16:38 PDT 2020
#
#  Note: This code was refactored from main()
#
#  Modifications:
#   Kathleen Biagas, Thu Aug 20 16:21:21 PDT 2020
#   Fix if-test for relative-to-tests-dir (remove not).
#
#   Kathleen Biagas, Fri Sep 11 10:10:44 PDT 2020
#   Restructure logic to handle more use-cases of file globbing.
#
# ----------------------------------------------------------------------------
def resolve_test_paths(tests,tests_dir):
    res = []

    if len(tests) == 1:
        # When run on linux command line, need to pass file globs in quotes
        # or the system tries to resolve the globs before passing to python,
        # which fails for relative paths (eg tests/plots/*.py).
        # Specifying more than one set, eg: "tests/plots/*.py tests/hybrid/*.py"
        # ends up as 1 item in the tests list, so split it up.
        tests = tests[0].split()

    for t in tests:
        t_abs_path = abs_path(t)
        # first check if we were passed the full path to a test script
        if os.path.isfile(t_abs_path):
            res.append(t_abs_path)
        else:
            # if not, assume it is relative to tests dir
            t_abs_path = abs_path(pjoin(tests_dir, "..",t))

            # is it a file
            if os.path.isfile(t_abs_path):
                res.append(t_abs_path)
            # or a glob
            elif '*' in os.path.basename(t_abs_path):
                matchcount=0
                for match in glob.iglob(t_abs_path):
                    if os.path.isfile(match):
                        res.append(match)
                        matchcount+=1
                if matchcount == 0:
                    print("[WARNING: could not find test files matching: {}]".format(t_abs_path))
            else:
                print("[WARNING: could not find test file: {}]".format(t_abs_path))

    return res

# ----------------------------------------------------------------------------
#  Method: main
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
#
#  Modifications:
#   Kathleen Biagas, Thu Feb  6 14:08:00 PST 2014
#   Only do ctest logging if ctest is enabled.
#
#   Burlen Loring, Fri Oct  2 09:56:26 PDT 2015
#   Make ctest output for each subtest. Moved the timer code
#   into the ctest module so that I could track time spent
#   in the sub test.
#
#   Kathleen Biagas, Wed Dec 18 17:22:59 MST 2019
#   On windows, glob any '*.py' tests names.
#
#
#   Cyrus Harrison, Fri Apr 10 08:57:27 PDT 2020
#   Allow passing test via full file path, in addition to rel to the 
#   tests dir.
#
#   Kathleen Biagas, Wed Jun  3 09:28:11 PDT 2020
#   Test for '*' on all platforms not just windows. Allows globbing from
#   batch scripts (where the command-line-glob won't necessarily be expanded).
#   And allows out-of-source testing to use file glob as well (surrounded by
#   quotes).
#
# ----------------------------------------------------------------------------
def main(opts,tests):
    """
    Main entry point for the test suite.
    """
    finalize_options(opts)
    Log("[[VisIt Test Suite]]")
    if opts["check_data"]:
        prepare_data_dir(opts["data_dir"])
    if opts["index"]:
        ridx  = opts["index"]
        tests = load_test_cases_from_index(opts["tests_dir"],ridx)
    elif opts["retry"]:
        Log("[Retrying failures from previous run]")
        ridx  = pjoin(opts["result_dir"],"results.json")
        tests = load_test_cases_from_index(opts["tests_dir"],ridx,True)
    elif len(tests) == 0:
        tests = find_test_cases(opts["tests_dir"],opts["classes"]) 
    # reckon test paths using helper
    tests = resolve_test_paths(tests,opts["tests_dir"])
    prepare_result_dirs(opts["result_dir"])
    ststamp = timestamp(sep=":")
    stime   = time.time()
    if opts["ctest"]:
        ctestInitTestTimer()
    Log("[Starting test suite run @ %s]" % ststamp)
    html_index = HTMLIndex(opts["result_dir"])
    html_index.write_header(opts["modes"],ststamp)
    json_index = JSONIndex(pjoin(opts["result_dir"],"results.json"))
    json_index.write_header(opts,tests,ststamp)
    error, results = launch_tests(opts,tests)
    etstamp = timestamp(sep=":")
    etime   = time.time()
    rtime   = str(math.ceil(etime - stime))
    html_index.write_footer(etstamp,rtime)
    json_index.finalize(etstamp,rtime)
    nskip   = len([ r.skip()  for r in results if r.skip() == True])
    Log("[Test suite run complete @ %s (wall time = %s)]" % (etstamp,rtime))
    if nskip > 0:
        Log("-- %d files due to skip list." % nskip)
    if not error:
        Log("++ Test suite run finished with NO errors.")
    else:
        nerrors = len([ r.error() for r in results if r.error() == True])
        if nerrors == 1:
            Log("!! Test suite run finished with %d error." % nerrors)
        else:
            Log("!! Test suite run finished with %d errors." % nerrors)
    if opts["cleanup"]:
        cleanup(opts["result_dir"],opts["cleanup_delay"])
    if opts["rsync_post"] is not None:
        rsync_post(opts["result_dir"],opts["rsync_post"])
    if opts['ctest']:
        if not error:
            sys.exit(0)
        else:
            sys.exit(1)
    return pjoin(opts["result_dir"],"results.json")


# ----------------------------------------------------------------------------
# ----------------------------------------------------------------------------
#  Method: run_visit_test and run_visit_tests
#
#  Programmer: Cyrus Harrison
#  Date:       Tue May 28 13:47:08 PDT 2013
#
#  Note: These are the two functions that comprise the module interface
# ----------------------------------------------------------------------------
# ----------------------------------------------------------------------------

def run_visit_test(script_file,
             data_dir=None,
             baseline_dir=None,
             output_dir=None,
             visit_bin=None,
             nprocs=None,
             verbose=False):
    tests = [script_file]
    return run_visit_tests(tests,
                           data_dir,
                           baseline_dir,
                           output_dir,
                           visit_bin,
                           nprocs,
                           verbose)

def run_visit_tests(tests,
                    data_dir=None,
                    baseline_dir=None,
                    output_dir=None,
                    visit_bin=None,
                    nprocs=None,
                    verbose=False):
    opts = default_suite_options()
    if not data_dir is None:
        opts["data_dir"]     = data_dir
    if not baseline_dir is None:
        opts["baseline_dir"] = baseline_dir
    if not output_dir is None:
        opts["result_dir"] = output_dir
    # Future: try to find "visit" in path?
    if not visit_bin is None:
        opts["executable"] = visit_bin
    # override other default options
    opts["check_data"]    = False
    opts["cleanup_delay"] = 1
    if verbose:
        opts["verbose"] = True
    if not nprocs is None:
        opts["nprocs"] = nprocs
    else:
        opts["nprocs"] = 1 # default to 1 for now
    opts["test_dir"] = os.path.split(os.path.abspath(__file__))[0]
    print(opts["test_dir"])
    res_file  = main(opts,tests)
    return JSONIndex.load_results(res_file,True)


def run_main():
    opts, tests = parse_args()
    main(opts,tests)

if __name__ == "__main__":
    run_main()
